import eia_backtestingimport pandas as pdimport plotly.graph_objects as goimport plotly.express as pximport osimport datetimeimport requestsimport json
Support for Torch based models not available. To enable them, install "darts", "u8darts[torch]" or "u8darts[all]" (with pip); or "u8darts-torch" or "u8darts-all" (with conda).
/opt/ai_dev_workshop/lib/python3.10/site-packages/statsforecast/core.py:27: TqdmExperimentalWarning:
Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)
series = pd.DataFrame(meta_json["series"])backtesting_output = {}for index, row in series.iterrows():print("Start backtesting for "+ row['subba_id']) df =None df =input[input["subba"] == row['subba_id']] df = df.sort_values(by = ["period"]) par = paramsfor i inrange(len(par)): par[i]["subba"] = row['subba_id'] backtesting_output[row['subba_id']] = eia_backtesting.backtesting(input= df, partitions=p, overlap = overlap, h = h, params = par, experiment_name = experiment_name +"_"+ row['subba_id'], mlflow_path = mlflow_path, overwrite =True, tags = tags)
Code
leaderboard =Nonebest =Nonefor index, row in series.iterrows(): subba = row["subba_id"]print(subba)print(backtesting_output[subba].leaderboard) leader_subba = backtesting_output[subba].leaderboard leader_subba["subba"] = subbaif leaderboard isNone: leaderboard = backtesting_output[subba].leaderboard best = leader_subba[leader_subba["mape"] == leader_subba["mape"].min()]else: leaderboard_temp = backtesting_output[subba].leaderboard leaderboard = leaderboard._append(leaderboard_temp) best = best._append(leader_subba[leader_subba["mape"] == leader_subba["mape"].min()])#best = best[["subba", "model","model_label" ,"mape", "rmse"]] best.to_csv(leaderboard_path, index =False) best
PGAE
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.079842 1135.350522
1 2024-06-14 model2 LinearRegressionModel 0.080266 1141.392395
2 2024-06-14 model3 LinearRegressionModel 0.057233 816.260224
3 2024-06-14 model4 LinearRegressionModel 0.061586 900.585172
4 2024-06-14 model5 LinearRegressionModel 0.084842 1216.658125
5 2024-06-14 model7 XGBModel 0.085270 1180.609964
coverage comments
0 0.808333 LM model with lags, training with 2 years of h...
1 0.804167 LM model with lags, training with 3 years of h...
2 0.829167 Model 2 with lag 1
3 0.808333 Model 1 with additional lags
4 0.789583 Model 1 with additional lags
5 0.733333 XGBoost with lags
SCE
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.053219 776.039709
1 2024-06-14 model2 LinearRegressionModel 0.053042 772.618437
2 2024-06-14 model3 LinearRegressionModel 0.046309 655.991958
3 2024-06-14 model4 LinearRegressionModel 0.044686 653.597728
4 2024-06-14 model5 LinearRegressionModel 0.068357 985.158385
5 2024-06-14 model7 XGBModel 0.069670 970.892884
coverage comments
0 0.929167 LM model with lags, training with 2 years of h...
1 0.929167 LM model with lags, training with 3 years of h...
2 0.889583 Model 2 with lag 1
3 0.904167 Model 1 with additional lags
4 0.893750 Model 1 with additional lags
5 0.789583 XGBoost with lags
SDGE
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.083208 173.564459
1 2024-06-14 model2 LinearRegressionModel 0.082176 173.083759
2 2024-06-14 model3 LinearRegressionModel 0.110862 206.756412
3 2024-06-14 model4 LinearRegressionModel 0.098167 189.345107
4 2024-06-14 model5 LinearRegressionModel 0.125380 235.072625
5 2024-06-14 model7 XGBModel 0.146632 269.623860
coverage comments
0 0.958333 LM model with lags, training with 2 years of h...
1 0.956250 LM model with lags, training with 3 years of h...
2 0.889583 Model 2 with lag 1
3 0.925000 Model 1 with additional lags
4 0.889583 Model 1 with additional lags
5 0.858333 XGBoost with lags
VEA
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.105364 18.354512
1 2024-06-14 model2 LinearRegressionModel 0.105227 18.476149
2 2024-06-14 model3 LinearRegressionModel 0.155049 26.586133
3 2024-06-14 model4 LinearRegressionModel 0.137781 22.932522
4 2024-06-14 model5 LinearRegressionModel 0.134385 23.258212
5 2024-06-14 model7 XGBModel 0.115111 21.245936
coverage comments
0 0.895833 LM model with lags, training with 2 years of h...
1 0.841667 LM model with lags, training with 3 years of h...
2 0.797917 Model 2 with lag 1
3 0.820833 Model 1 with additional lags
4 0.854167 Model 1 with additional lags
5 0.879167 XGBoost with lags
forecast_label
model_label
model
mape
rmse
coverage
comments
subba
2
2024-06-14
model3
LinearRegressionModel
0.057233
816.260224
0.829167
Model 2 with lag 1
PGAE
3
2024-06-14
model4
LinearRegressionModel
0.044686
653.597728
0.904167
Model 1 with additional lags
SCE
1
2024-06-14
model2
LinearRegressionModel
0.082176
173.083759
0.956250
LM model with lags, training with 3 years of h...
SDGE
1
2024-06-14
model2
LinearRegressionModel
0.105227
18.476149
0.841667
LM model with lags, training with 3 years of h...
VEA
Code
print(leaderboard)
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.079842 1135.350522
1 2024-06-14 model2 LinearRegressionModel 0.080266 1141.392395
2 2024-06-14 model3 LinearRegressionModel 0.057233 816.260224
3 2024-06-14 model4 LinearRegressionModel 0.061586 900.585172
4 2024-06-14 model5 LinearRegressionModel 0.084842 1216.658125
5 2024-06-14 model7 XGBModel 0.085270 1180.609964
0 2024-06-14 model1 LinearRegressionModel 0.053219 776.039709
1 2024-06-14 model2 LinearRegressionModel 0.053042 772.618437
2 2024-06-14 model3 LinearRegressionModel 0.046309 655.991958
3 2024-06-14 model4 LinearRegressionModel 0.044686 653.597728
4 2024-06-14 model5 LinearRegressionModel 0.068357 985.158385
5 2024-06-14 model7 XGBModel 0.069670 970.892884
0 2024-06-14 model1 LinearRegressionModel 0.083208 173.564459
1 2024-06-14 model2 LinearRegressionModel 0.082176 173.083759
2 2024-06-14 model3 LinearRegressionModel 0.110862 206.756412
3 2024-06-14 model4 LinearRegressionModel 0.098167 189.345107
4 2024-06-14 model5 LinearRegressionModel 0.125380 235.072625
5 2024-06-14 model7 XGBModel 0.146632 269.623860
0 2024-06-14 model1 LinearRegressionModel 0.105364 18.354512
1 2024-06-14 model2 LinearRegressionModel 0.105227 18.476149
2 2024-06-14 model3 LinearRegressionModel 0.155049 26.586133
3 2024-06-14 model4 LinearRegressionModel 0.137781 22.932522
4 2024-06-14 model5 LinearRegressionModel 0.134385 23.258212
5 2024-06-14 model7 XGBModel 0.115111 21.245936
coverage comments subba
0 0.808333 LM model with lags, training with 2 years of h... PGAE
1 0.804167 LM model with lags, training with 3 years of h... PGAE
2 0.829167 Model 2 with lag 1 PGAE
3 0.808333 Model 1 with additional lags PGAE
4 0.789583 Model 1 with additional lags PGAE
5 0.733333 XGBoost with lags PGAE
0 0.929167 LM model with lags, training with 2 years of h... SCE
1 0.929167 LM model with lags, training with 3 years of h... SCE
2 0.889583 Model 2 with lag 1 SCE
3 0.904167 Model 1 with additional lags SCE
4 0.893750 Model 1 with additional lags SCE
5 0.789583 XGBoost with lags SCE
0 0.958333 LM model with lags, training with 2 years of h... SDGE
1 0.956250 LM model with lags, training with 3 years of h... SDGE
2 0.889583 Model 2 with lag 1 SDGE
3 0.925000 Model 1 with additional lags SDGE
4 0.889583 Model 1 with additional lags SDGE
5 0.858333 XGBoost with lags SDGE
0 0.895833 LM model with lags, training with 2 years of h... VEA
1 0.841667 LM model with lags, training with 3 years of h... VEA
2 0.797917 Model 2 with lag 1 VEA
3 0.820833 Model 1 with additional lags VEA
4 0.854167 Model 1 with additional lags VEA
5 0.879167 XGBoost with lags VEA